#!/bin/bash
set -x
Cur_day=`date -d '6 day ago' +%Y%m%d`
TIME_WAIT_MAX=36000
TIME_SLEEP=300
#运行时传城市参数进来 $1
city=$1
city_today=$1"_$Cur_day"
#city_today=$1
dbname=$2
MYPATHS=`pwd`
Local_inputdata_path=$MYPATHS/$1
#Local_inputdata_path=$MYPATHS
Local_code_path=$MYPATHS
#Local_hadoop_path=/usr/local/apps/hadoop-2.6.4/share/hadoop/tools/lib
Local_hadoop_path=/alidata/zhuge/cloudera/parcels/CDH-5.9.0-1.cdh5.9.0.p0.23/lib
Local_hadoop_jar=/alidata/zhuge/cloudera/parcels/CDH-5.9.0-1.cdh5.9.0.p0.23/jars/hadoop-streaming-2.6.0-cdh5.9.0.jar

start_time=`date +%s`
#删除文件，如果存在的话
<<!
python suspend.py $1 sell
if [ $? -ne 0 ]; then
    exit 0
fi
!
#运行时传该城市对应的数据库名作为第二个参数
#rm -rf $Local_inputdata_path/house_sell_gov.gz
#python $Local_code_path/dumpdata.py $dbname $city
#python $Local_code_path/dumpdata.py $dbname $city
#gzip $Local_inputdata_path/house_sell_gov
hadoop fs -rmr /Compare/sell/$city_today/*
hadoop fs -mkdir /Compare/sell/$city_today
#hadoop fs -copyFromLocal $Local_inputdata_path/house_sell_gov.gz /Compare/sell/$city_today
mkdir $Local_inputdata_path/$city_today/
mkdir $Local_inputdata_path/$city_today/house_sell
mkdir $Local_inputdata_path/$city_today/house_sell_gov
mkdir $Local_inputdata_path/$city_today/house_sell_info
mkdir $Local_inputdata_path/$city_today/house_sell_pic
#mapred3:$
hadoop fs -rmr /Compare/sell/$city_today/mapred3
hadoop jar $Local_hadoop_jar \
-input /ETL/sell/$city_today/mapred9 \
-output  /Compare/sell/$city_today/mapred3 \
-mapper '/usr/bin/php mapper_3.php' \
-reducer '/usr/bin/php reducer3_1.php' \
-file "$Local_code_path/mapper_3.php" \
-file "$Local_code_path/reducer3_1.php"
#mapred3_2
hadoop fs -rmr /Compare/sell/$city_today/mapred3_2
hadoop jar $Local_hadoop_jar \
-D map.output.key.field.separator=. \
-D num.key.fields.for.partition=1 \
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-input /Compare/sell/$city_today/mapred3 \
-output  /Compare/sell/$city_today/mapred3_2 \
-mapper 'cat' \
-reducer '/usr/bin/php reducer3_2.php' \
-file "$Local_code_path/reducer3_2.php"
#mapred4
hadoop fs -rmr /Compare/sell/$city_today/mapred4
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /Compare/sell/$city_today/mapred3_2/part-* \
-output /Compare/sell/$city_today/mapred4 \
-mapper '/usr/bin/php mapper_4.php' \
-file "$Local_code_path/mapper_4.php" 
#step5 sql file(house_sell_info)
hadoop fs -rmr /Compare/sell/$city_today/mapred5
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /Compare/sell/$city_today/mapred4/part-* \
-output /Compare/sell/$city_today/mapred5 \
-mapper '/usr/bin/php mapper5.php' \
-file "$Local_code_path/mapper5.php"
rm -rf $Local_inputdata_path/$city_today/house_sell_info/*
hadoop fs -copyToLocal /Compare/sell/$city_today/mapred5/* $Local_inputdata_path/$city_today/house_sell_info
hadoop fs -getmerge /Compare/sell/$city_today/mapred5/part* $Local_inputdata_path/$city_today/house_sell_info/house_sell_info
#step6 sql file(house_sell)
hadoop fs -rmr /Compare/sell/$city_today/mapred6
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /Compare/sell/$city_today/mapred4/part-* \
-output /Compare/sell/$city_today/mapred6 \
-mapper '/usr/bin/php mapper6.php' \
-file "$Local_code_path/mapper6.php"
rm -rf $Local_inputdata_path/$city_today/house_sell/*
hadoop fs -copyToLocal /Compare/sell/$city_today/mapred6/* $Local_inputdata_path/$city_today/house_sell
hadoop fs -getmerge /Compare/sell/$city_today/mapred6/part* $Local_inputdata_path/$city_today/house_sell/house_sell
#step7 sql file(house_sell_gov)
hadoop fs -rmr /Compare/sell/$city_today/mapred7
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /Compare/sell/$city_today/mapred4/part-* \
-output /Compare/sell/$city_today/mapred7 \
-mapper '/usr/bin/php mapper7_2_20.php' \
-file "$Local_code_path/mapper7_2_20.php"
rm -rf $Local_inputdata_path/$city_today/house_sell_gov/*
hadoop fs -copyToLocal /Compare/sell/$city_today/mapred7/* $Local_inputdata_path/$city_today/house_sell_gov
hadoop fs -getmerge /Compare/sell/$city_today/mapred7/part* $Local_inputdata_path/$city_today/house_sell_gov/house_sell_gov

end_time=`date +%s`
echo $(($end_time-$start_time))
#step 8(house_sell_pic)
hadoop fs -rmr /Compare/sell/$city_today/mapred8
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /Compare/sell/$city_today/mapred4/part-* \
-output /Compare/sell/$city_today/mapred8 \
-mapper '/usr/bin/php mapper8.php' \
-file "$Local_code_path/mapper8.php" \
-file "$Local_code_path/upyun.class.php"
#copyToLocal
rm -rf $Local_inputdata_path/$city_today/house_sell_pic/*
hadoop fs -copyToLocal /Compare/sell/$city_today/mapred8/* $Local_inputdata_path/$city_today/house_sell_pic
hadoop fs -getmerge /Compare/sell/$city_today/mapred8/part* $Local_inputdata_path/$city_today/house_sell_pic/house_sell_pic
<<!
dir="../data/sell/$city_today/house_sell_gov/"
php load.php $dbname $dir 
dir="../data/sell/$city_today/house_sell_info/"
php load.php $dbname $dir 
dir="../data/sell/$city_today/house_sell/"
php load.php $dbname $dir 
python datacheck.py $dbname $city
if [ $? -ne 0 ]; then
    exit 0
fi
sh switch.sh $dbname offline
if [ $? -ne 0 ]; then
    exit 0
fi
python notify_es.py $city sell offline

dir="$Local_inputdata_path/$city_today/house_sell_gov/"
php load_online.php $dbname $dir
dir="$Local_inputdata_path/$city_today/house_sell_info/"
php load_online.php $dbname $dir
dir="$Local_inputdata_path/$city_today/house_sell/"
php load_online.php $dbname $dir

python datacheck_online.py $dbname $city
if [ $? -ne 0 ]; then
    exit 0
fi
!
<<!
sh switch.sh $dbname online
!
dir="/alidata/home/pangyong/zhuge_offline_compute/Compare/sell/$city_today/$city_today/house_sell_gov/"
php load_online.php $dbname $dir
dir="/alidata/home/pangyong/zhuge_offline_compute/Compare/sell/$city_today/$city_today/house_sell_info/"
php load_online.php $dbname $dir
dir="/alidata/home/pangyong/zhuge_offline_compute/Compare/sell/$city_today/$city_today/house_sell/"
php load_online.php $dbname $dir

python datacheck_online.py $dbname $city
if [ $? -ne 0 ]; then
    exit 0
fi
!
<<!
sh switch.sh $dbname online
if [ $? -ne 0 ]; then
    exit 0
fi
!
